# coding: UTF-8
import os
import re
import time
import torch
import jieba
import numpy as np
import pickle as pkl
from tqdm import tqdm
from datetime import timedelta

MAX_VOCAB_SIZE = 10000  # 词表长度限制
NUM, STR, UNK, PAD = '<NUM>', '<STR>', '<UNK>', '<PAD>'  # 数字, 字母, 未知字, padding符号
stopwords_path = './source/rule/stopwords.txt'
userdict_filepath = './source/rule/all_js_keywords.txt'

def isletter(str):
    my_re = re.compile(r'[A-Za-z]',re.S)
    res = re.findall(my_re, str)
    return len(res) == len(str)

def read_from_txt(path):
    from itertools import islice
    sequence = []
    text_labels = []
    texts = []
    with open(path, 'r', encoding='UTF-8') as f:
        # Text_labels after text and a space is in between
        print('read from ', path)
        for line in tqdm(islice(f, 1, None)): # 跳过首行
            lin = line.strip()
            try:
                l, t, s= lin.split('\t')
                seq = s.split(' ')
            except:
                print(lin)
                input()
            sequence.append(seq)
            text_labels.append(int(l))
            texts.append(t)
    return sequence, text_labels, texts

def convert2tensor(texts, labels, tokenizer, vocab, device, oversamp='false', pad_size=32):
    # 把所有句子用<PAD>补长到pad_size, 超过则截短到pad_size
    contents = []
    lengths = []
    source = []
    for text in tqdm(texts):
        source.append(text)
        tokens = tokenizer(text)
        #print(tokens)
        seq_len = len(tokens)
        if len(tokens) < pad_size:
            tokens.extend([PAD] * (pad_size - len(tokens)))
        else:
            tokens = tokens[:pad_size]
            seq_len = pad_size
        words_line = []
        for token in tokens:
            words_line.append(vocab.get(token, vocab.get(UNK)))
        contents.append(words_line)
        lengths.append(seq_len)
    '''
    from collections import Counter
    print(Counter(labels))
    '''
    contents = torch.LongTensor(contents).to(device)
    labels = torch.LongTensor(labels).to(device)
    lengths = torch.LongTensor(lengths).to(device)
    return contents, labels, lengths, source

def get_time_dif(start_time):
    end_time = time.time()
    time_dif = end_time - start_time
    return timedelta(seconds=int(round(time_dif)))

def tokenizer(text):
    '''分词+去停用词'''
    jieba.load_userdict(userdict_filepath)
    stopwords = get_stopwords_list()
    sentence_depart = seg_depart(text)
    sentence_depart = move_stopwords(sentence_depart, stopwords)
    return sentence_depart

def get_stopwords_list():
    stopwords = [line.strip() for line in open(stopwords_path, encoding='UTF-8').readlines()]
    return stopwords

def seg_depart(sentence):
    sentence_depart = jieba.lcut(sentence.strip(), cut_all=False, HMM=True)
    return sentence_depart

def remove_digits(input_str):
    punc = u'0123456789.'
    output_str = re.sub(r'[{}]+'.format(punc), '', input_str)
    return output_str

def move_stopwords(sentence_list, stopwords_list):
    out_list = []
    for word in sentence_list:
        if word not in stopwords_list:
            if not remove_digits(word):
                continue
            if word != '\t':
                out_list.append(word)
    return out_list

def cut_sent(para):
    para = re.sub('([。！;；？\?])([^”’])', r"\1\n\2", para)  # 单字符断句符
    para = re.sub('(\.{6})([^”’])', r"\1\n\2", para)  # 英文省略号
    para = re.sub('(\…{2})([^”’])', r"\1\n\2", para)  # 中文省略号
    para = re.sub('([。！？\?][”’])([^，。！？\?])', r'\1\n\2', para)
    # 如果双引号前有终止符，那么双引号才是句子的终点，把分句符\n放到双引号后，注意前面的几句都小心保留了双引号
    para = para.rstrip()  # 段尾如果有多余的\n就去掉它
    # 很多规则中会考虑分号;，但是这里我把它忽略不计，破折号、英文双引号等同样忽略，需要的再做些简单调整即可。
    return para.split("\n")


if __name__ == "__main__":
    sentence = '1、判令被告赵军霞偿还原。/告借款本息及应收费用共计4278.6元（计算至2017年1月10日，实际还款额以合同约定的计费方式计算至最终还款日）'
    print(cut_sent(sentence))
    input()
    '''
    tokenizer = lambda x: [y for y in x]
    print(build_vocab('news.txt', tokenizer, MAX_VOCAB_SIZE, 2))
    
    # 下面的目录、文件名按需更改。
    train_dir = "./THUCNews/data/train.txt"
    vocab_dir = "./THUCNews/data/vocab.pkl"
    pretrain_dir = "./THUCNews/data/sgns.sogou.char"
    emb_dim = 300
    filename_trimmed_dir = "./THUCNews/data/embedding_SougouNews"
    if os.path.exists(vocab_dir):
        word_to_id = pkl.load(open(vocab_dir, 'rb'))
    else:
        # tokenizer = lambda x: x.split(' ')  # 以词为单位构建词表(数据集中词之间以空格隔开)
        tokenizer = lambda x: [y for y in x]  # 以字为单位构建词表
        word_to_id = build_vocab(train_dir, tokenizer=tokenizer, max_size=MAX_VOCAB_SIZE, min_freq=1)
        pkl.dump(word_to_id, open(vocab_dir, 'wb'))

    embeddings = np.random.rand(len(word_to_id), emb_dim)
    f = open(pretrain_dir, "r", encoding='UTF-8')
    for i, line in enumerate(f.readlines()):
        # if i == 0:  # 若第一行是标题，则跳过
        #     continue
        lin = line.strip().split(" ")
        if lin[0] in word_to_id:
            idx = word_to_id[lin[0]]
            emb = [float(x) for x in lin[1:301]]
            embeddings[idx] = np.asarray(emb, dtype='float32')
    f.close()
    np.savez_compressed(filename_trimmed_dir, embeddings=embeddings)
    '''
    
